{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.5505782,  0.       ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCarContinuous-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        reward = -1.0\n",
    "        if done:\n",
    "            reward = 100\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhg0lEQVR4nO3df1AU9/0/8Oce94M74I7fdyKgJhLx9w9AOEETlUgoaWJif6WZaSbtJGOCnZhkMhNmvjHTmXbotDOdNm3GzLTTJJ3p1I6d2iZWSRAURI7fEhENaqKCcnegyB3Icb/29f3DsB9P0fBLlrt7PWZ2DLvvu3vdwj6z+97d9wpERGCMMRko5C6AMRa+OIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7LhAGKMyYYDiDEmGw4gxphsOIAYY7KRLYDef/99LFy4EJGRkcjNzUVTU5NcpTDGZCJLAP3zn//EG2+8gXfffRdtbW1YvXo1ioqK0NfXJ0c5jDGZCHLcjJqbm4ucnBz86U9/AgCIooi0tDT8/Oc/x9tvvz3b5TDGZKKc7Q/0eDxobW1FWVmZNE+hUKCwsBAWi2Xc17jdbrjdbulnURQxMDCAhIQECILwwGtmjE0OEWFoaAgpKSlQKO59oDXrAXTt2jX4/X4YjcaA+UajEV9++eW4rykvL8cvfvGL2SiPMTaDenp6kJqaes/lsx5AU1FWVoY33nhD+tnhcCA9PR09PT3Q6/UyVsYYG4/T6URaWhpiYmLu227WAygxMRERERGw2+0B8+12O0wm07iv0Wg00Gg0d83X6/UcQIzNYd/WRTLrZ8HUajWysrJQVVUlzRNFEVVVVTCbzbNdDmNMRrIcgr3xxht44YUXkJ2djfXr1+P3v/89bt68iRdffFGOchhjMpElgH74wx+iv78fe/bsgc1mw5o1a1BRUXFXxzRjLLTJch3QdDmdThgMBjgcDu4DYmwOmug2GhRnwRhjD57HcxV+/w2oVGmIiNABUD7w6+w4gBhjAIDBwf+gt/f/QalMRmRkJnS61dBqVyEycjlUKiMiIgwYO281U8HEAcQY+4YIv38Qfv8g3O5zcDg+AaCEQhEJtToNGs1iREWth15fhKionBn5RA4gxtg9KKBUxkKpnAetdrm0R6RSpczYJ3AAMcYAAF6vEpGRa6DVLv0mbNZCo1kElSoFCkUkAMWM9wlxADHGAABtbUnYsOG/iI+fD0GImJXP5BERGWMAAK9XAUGInrXwATiAGGMy4gBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmG7wVjjH2r2wdOJaKAm1Knc4MqBxBjLAARwel0Ynh4GFarFf39/RgdHYXD4YBWq8Xw8DC0Wi1UKhUiIyMRHx+P6OhozJ8/HzExMVCr1RP+LA4gxsIcEcHr9WJgYACfffYZRFGEy+WCyWRCUlISFi9ejLi4OKhUqoDHLIuiiNHRUQwMDMBqteLixYtwOByIiopCXFzchD6bA4ixMEREEEURVqsVJ0+exLVr1xAZGYmsrCykp6cHPAj0fodYBoMBRqMRmZmZ0rzBwUF0dXVNqA4OIMbCjNvtxrlz59DS0gKTyYR169YhOTkZSuXUB6G//XVxcXFYtmzZhF7HAcRYGBg7zOro6MDp06eRmZmJ73//+4iOjpa1Lg4gxkKc3+9HZ2cnOjo6sHTpUjz33HNQqVQP/JE7E8EBxFiIIiJcvXoVzc3NSExMxPbt26HT6eZE8IzhAGIsBHk8HjQ1NcFut2PLli2IjY2dU8EzhgOIsRBCROjt7cXRo0eRnZ0Ns9mMiIjZG+N5sjiAGAsRfr8f7e3tsNvteOqppxATEzMn93puN+l7wWpra/Hd734XKSkpEAQB//nPfwKWExH27NmDefPmQavVorCwEOfPnw9oMzAwgOeffx56vR6xsbH42c9+huHh4Wl9EcbC2c2bN/HZZ59BEAQUFRVBr9fP+fABphBAN2/exOrVq/H++++Pu/w3v/kN3nvvPXzwwQdobGxEVFQUioqKMDo6KrV5/vnn0dnZicrKShw8eBC1tbV4+eWXp/4tGAtTRISenh4cOXIEa9euxdq1a+f0IdddaBoA0IEDB6SfRVEkk8lEv/3tb6V5g4ODpNFo6B//+AcREZ05c4YAUHNzs9Tm8OHDJAgCXb16dUKf63A4CAA5HI7plM9YUBNFkb788kv617/+RcPDwySKotwlSSa6jc7ocBwXL16EzWZDYWGhNM9gMCA3NxcWiwUAYLFYEBsbi+zsbKlNYWEhFAoFGhsbx31ft9sNp9MZMDEWznw+HywWC/r7+/HUU08hKioqKA657jSjAWSz2QAARqMxYL7RaJSW2Ww2JCcnByxXKpWIj4+X2typvLwcBoNBmtLS0maybMaCitfrxfHjx6FSqZCfnw+VSiV3SVMWFAOSlZWVweFwSFNPT4/cJTE264gIo6OjqKmpQXp6OrKzs4Nyr+d2MxpAJpMJAGC32wPm2+12aZnJZEJfX1/Acp/Ph4GBAanNnTQaDfR6fcDEWDghIrjdblRWVmLJkiV46KGHgj58gBkOoEWLFsFkMqGqqkqa53Q60djYCLPZDAAwm80YHBxEa2ur1Ka6uhqiKCI3N3cmy2EsZNy8eRPV1dXIy8tDampqSIQPMIULEYeHh3HhwgXp54sXL6K9vR3x8fFIT0/H7t278ctf/hIZGRlYtGgR3nnnHaSkpGD79u0AgKVLl+KJJ57ASy+9hA8++ABerxe7du3Cj370I6SkpMzYF2MsFBARhoeHUVVVhU2bNiE+Pl7ukmbWZE+vHT16lADcNb3wwgtEdOvU4DvvvENGo5E0Gg1t3bqVurq6At7j+vXr9Nxzz1F0dDTp9Xp68cUXaWhoaMI18Gl4Fi6cTicdPnyYbty4MadOs3+biW6jAtFto00HCafTCYPBAIfDwf1BLCQREQYHB3HixAkUFBQgNjZW7pImZaLbaFCcBWMsnBARRkZGcOLECZjNZhgMBrlLemD4ZlTG5pjR0VEcO3YMGzZsCL0+nzvwHhBjc4jb7UZVVRWys7Mn/GSJYMYBxNgcMXaF85o1a5CcnBwyp9rvhwOIsTnA6/WiqqoKDz30EObPnx8W4QNwADEmOyJCU1MTTCYTFi1aFDbhA3AAMSYrIkJHRwfUajVWr14dVuEDcAAxJhsiwunTp9Hb24usrKywCx+AA4gx2VitVvT29mLLli0Bz1wPJ+H5rRmT2dgN2Zs2bYJarZa7HNlwADE2y1wuFw4dOgSz2YzIyEi5y5FVUF8JHYS3sbEw5/f7UVtbi0cffRSJiYlylyO7oN4DcjqdHEIsaIiiCIvFgtTUVB565htBHUD19fXweDxyl8HYtyIinD17Fh6PB8uWLQvLM17jCeoAyszMRFVVFXw+n9ylMHZPRAS73Y6enh5s3LiRw+c2QR1ACxcuRGJiItrb2/lQjM1Zw8PDsFgs2LRpU1A/weJBCOoAEgQBWVlZcDqduHz5MocQm3O8Xi+OHTuGnJwcaLVaucuZc4I6gAAgIiICBQUFaGlpwY0bN+QuhzGJKIpoaGjAI488ElY3mE5G0AcQAKhUKjz22GM4fvw43G633OUwJt1moVAokJGRweFzDyERQIIgICEhAStXrkR9fT1EUZS7JBbGiAj9/f24ePEicnNzw/Y2i4kImTUjCAIWLVqEqKgonDx5kvuDmGxcLhcaGhqwZcsWKJVBfa3vAxcyAQTcCqF169bBZrPBZrNxCLFZ5/P5UF1djXXr1iEmJkbucua8kAogAFAqldi8eTMaGxvhcrnkLoeFESJCS0sLkpKSMH/+fLnLCQohF0AAoNVqkZOTgyNHjsDr9cpdDgsDRIQrV65gaGgI2dnZ3Ok8QSEZQIIgICUlBSkpKdwfxGaF0+lEU1MTNm7ciIiICLnLCRohGUDArRBau3YthoaG0N3dLXc5LIT5fD5UVVUhNzcXGo1G7nKCSsgGEHDrIsX8/Hy0tLRgYGCA94TYjBvr93nkkUeQmprKh16TFNIBBAAajQZmsxk1NTXw+/1yl8NCzFdffYWRkREsW7ZM7lKC0qQCqLy8HDk5OYiJiUFycjK2b9+Orq6ugDajo6MoLS1FQkICoqOjsWPHDtjt9oA23d3dKCkpgU6nQ3JyMt56660Hdke7IAiYN28eMjIy0NLSwntBbMYMDg6iqakJeXl5fLHhFE1qrdXU1KC0tBQNDQ2orKyE1+vFtm3bcPPmTanN66+/jk8//RT79+9HTU0Nent78eyzz0rL/X4/SkpK4PF4UF9fj48//hgfffQR9uzZM3Pf6g6CIGD58uUYHh7GV199xSHEps3r9aKurg6PP/44dDqd3OUEL5qGvr4+AkA1NTVERDQ4OEgqlYr2798vtTl79iwBIIvFQkREhw4dIoVCQTabTWqzd+9e0uv15Ha7J/S5DoeDAJDD4ZhUvSMjI/T3v/+dnE7npF7H2O1EUaQTJ05QZ2cniaIodzlz0kS30WntNzocDgBAfHw8AKC1tRVerxeFhYVSm8zMTKSnp8NisQAALBYLVq5cCaPRKLUpKiqC0+lEZ2fnuJ/jdrvhdDoDpqnQarUoLCxEbW0tD2LGpoSIcP78eVy/fh2ZmZnc6TxNUw4gURSxe/du5OfnY8WKFQAAm80GtVqN2NjYgLZGoxE2m01qc3v4jC0fWzae8vJyGAwGaUpLS5tq2UhKSkJaWhr3B7EpGRoawpkzZ/D4449zv88MmPIaLC0txenTp7Fv376ZrGdcZWVlcDgc0tTT0zPl9xIEAStWrEB/fz8uXbrEIcQmzOfzoba2Fvn5+WH/OJ2ZMqUA2rVrFw4ePIijR48iNTVVmm8ymeDxeDA4OBjQ3m63w2QySW3uPCs29vNYmztpNBro9fqAaToUCgUKCwtx6tQpDA8PT+u9WHggItTW1iIlJYUfpzODJhVARIRdu3bhwIEDqK6uxqJFiwKWZ2VlQaVSoaqqSprX1dWF7u5umM1mAIDZbEZHRwf6+vqkNpWVldDr9bN6LYVWq0Vubi7q6uq4P4jdFxGhu7sbPp8Pa9as4X6fmTSZnu1XXnmFDAYDHTt2jKxWqzSNjIxIbXbu3Enp6elUXV1NLS0tZDabyWw2S8t9Ph+tWLGCtm3bRu3t7VRRUUFJSUlUVlY24z3s30YURWptbaUTJ07w2Qx2T0NDQ/Tvf/+bXC6X3KUEjYluo5MKIADjTh9++KHUxuVy0auvvkpxcXGk0+nomWeeIavVGvA+ly5douLiYtJqtZSYmEhvvvkmeb3eGf9yE+Hz+ejzzz+n7u7uab8XCz0ej4f2799Pvb29cpcSVCa6jQpEwdcL63Q6YTAY4HA4pt0fBNwawa6iogLbtm2DTqfjXWwG4NahV2trqzTQHf9dTNxEt1E+jwggMjIS69evx5EjR3g8aSa5cuUKbty4wf0+DxAHEP5v/KDU1FR88cUXfGqewel0orGxEfn5+Ty+zwPEAfQNQRCwZs0a2O12WK1WucthMvL7/airq0NeXh4/TPAB4wC6TUREBDZt2oS6urqAG2xZ+CAitLe3Y968efwwwVnAAXQHnU4nXR/E/UHhhYjQ09OD/v5+rFq1isNnFnAA3UEQBKSnpyMhIQEdHR3cHxRGXC4XGhsbeVznWcQBNI6x8aSvXLmCq1evcgiFAb/fj9raWmzYsAFRUVFylxM2OIDuISIiAgUFBWhqauLni4U4IsIXX3yB5ORkpKSkyF1OWOEAug+DwYCcnBzU1dXxeNIhiojQ29uLK1euYPXq1dzvM8s4gL5FamoqEhIS0NbWxodiIcjlcqG5uRlbt27l8X1kwGv8WwiCgNWrV8Nms6G3t5dDKIT4fD4cP34c69evR1RUFO/9yIADaAKUSiW2bt3K/UEhhL55nldCQgLmzZsndzlhiwNogsbGDzp27BiPHxQCenp6MDw8jLVr1/Kej4w4gCZo7PliSUlJ/Lz5IEZEGBoaQltbGwoKCvh6H5lxAE3C2LAMTqeTnzcfpPx+Pw4dOoS8vDx+jvscwAE0SWPPm29vb8eNGzd4TyiIiKKIpqYmrFy5EiaTiQ+95gAOoCnQaDTIz89HTU0N9wcFCSLChQsX4PP5kJmZKXc57BscQFMgCAISEhKwfPlyWCwWvml1jiMi9PX1oaWlhZ/jPsfwb2KKBEHA4sWLERERgc7OTj4Um8PcbjcaGhrw5JNPQq1Wy10Ouw0H0DQIgoD169fj3LlzPIjZHOX3+3H8+HGsW7cOMTExcpfD7sABNE0qlQpPPPEEmpub+SGHc8zYoPKxsbFITU3lTuc5iANoBuh0OpjNZtTU1MDr9cpdDsOt8Pn666/hcrn4iRZzGAfQDBAEAUlJSVi4cCEaGhq4U1pmRITr16/jzJkz2LBhA19sOIdxAM0QQRCwbNkyaDQa7pSWmcvlwmeffYaCggIolUq5y2H3wQE0gwRBQHZ2Nnp6enDp0iUOIRn4fD7U1tZiy5YtiIuL40OvOY4DaIYpFAo8+uijOHnyJF8pPctEUUR9fT0WLlwIk8kkdzlsAjiAHoCoqCgUFhairq4Oo6OjcpcTFogIHR0diIiIwJIlS3jPJ0hMKoD27t2LVatWQa/XQ6/Xw2w24/Dhw9Ly0dFRlJaWIiEhAdHR0dixYwfsdnvAe3R3d6OkpAQ6nQ7Jycl46623QvJ2hpiYGGRlZaGiogIej0fuckIaEeHSpUsYGBhAXl4eh08QmVQApaam4te//jVaW1vR0tKCLVu24Omnn0ZnZycA4PXXX8enn36K/fv3o6amBr29vXj22Wel1/v9fpSUlMDj8aC+vh4ff/wxPvroI+zZs2dmv9UcMPa456VLl6K+vp7PjD0gRASbzYbOzk5+jHIwommKi4ujv/zlLzQ4OEgqlYr2798vLTt79iwBIIvFQkREhw4dIoVCQTabTWqzd+9e0uv15Ha7J/yZDoeDAJDD4Zhu+Q+cKIrU1tZGLS0tJIqi3OWEHKfTSf/9739peHhY7lLYbSa6jU65D8jv92Pfvn24efMmzGYzWltb4fV6UVhYKLXJzMxEeno6LBYLAMBisWDlypUwGo1Sm6KiIjidTmkvKtSMjSk9MDCAU6dOcaf0DBoZGcHx48exceNGfpZXkJp0AHV0dCA6OhoajQY7d+7EgQMHsGzZMthsNqjVasTGxga0NxqNsNlsAACbzRYQPmPLx5bdi9vthtPpDJiCiUKhwObNmzEwMMCn52eI2+1GRUUFVq9efdffHAsekw6gJUuWoL29HY2NjXjllVfwwgsv4MyZMw+iNkl5eTkMBoM0paWlPdDPexCUSiUKCgpw9uxZ2Gw2DqFp8Pv9qKurw7p16zB//nzudA5ikw4gtVqNxYsXIysrC+Xl5Vi9ejX+8Ic/wGQywePxYHBwMKC93W6XrskwmUx3nRUb+/l+122UlZXB4XBIU09Pz2TLnhNUKhUee+wxtLS04Pr163KXE5TGwic1NRULFiyQuxw2TdO+DkgURbjdbmRlZUGlUqGqqkpa1tXVhe7ubpjNZgCA2WxGR0cH+vr6pDaVlZXQ6/VYtmzZPT9Do9FIp/7HpmCl0+mwefNmNDc348aNG3KXE1REUURLSwu0Wi0eeeQR3vMJBZPp2X777beppqaGLl68SKdOnaK3336bBEGgzz//nIiIdu7cSenp6VRdXU0tLS1kNpvJbDZLr/f5fLRixQratm0btbe3U0VFBSUlJVFZWdkD6WGfq0RRpMHBQdq3bx8NDAzw2bEJ8Pv91NLSQu3t7eT3++Uuh32LiW6jkwqgn/70p7RgwQJSq9WUlJREW7dulcKHiMjlctGrr75KcXFxpNPp6JlnniGr1RrwHpcuXaLi4mLSarWUmJhIb775Jnm93smUEfQBNOb69etUUVFBTqdT7lLmNL/fT/X19XTixAkOnyAx0W1UIAq+3lCn0wmDwQCHwxHUh2MAcOPGDRw/fhwFBQV88+Q4RFFER0cHPB4PsrKyeDznIDHRbZR/mzKLjY2F2WxGfX09hoaG5C5nThFFEQ0NDfB6vRw+IYp/ozIbG8wsPz8f9fX1GBgY4FP0uBU+bW1tAIB169Zx+IQo/q3OEbGxscjNzcWRI0fC/hS93+9HQ0MDVCoVP0YnxPFvdo4QBAFxcXEoLi5Gc3MzrFZrWO4Jeb1e1NTUQKPRYNWqVRw+IY5/u3NMdHQ0Nm/ejLa2Nly4cCGsQujmzZuorq7GwoULeSD5MMEBNMcIgoDIyEhs27YNVqsVp06dCvmhPIgIDocDhw8fxtKlS7Fo0SIOnzDBp+HnML/fj6amJrhcLmzcuBEqlWrK72W1WvHee+8hJiYGCQkJiI+PR0JCAgwGA3Q6HXQ6HbRaLXQ6HTQaDRQKhRQCd/47k4gIly9fxqlTp1BQUID4+PgZ/ww2+ya6jfIjA+awiIgI5OXl4auvvkJlZSUKCgoQExMzpSD4+uuv8bvf/Q4ejweCIEjvoVQqpeAZ+1en0wWEVHx8PJYsWYIf/OAHM9onI4oiOjs7ce7cOZSUlECj0czYe7PgwAE0xwmCgIcffhh6vR4VFRXIy8tDWlrapEPo4sWL8Pv9AG7tdYzt+Ho8Hng8Hjgcjvu+PicnB9/73vdmJICICG63G3V1ddDr9Xj66af58TlhivuAgsDYtUJPPfUUurq6UFdXN+lxtM+fPy8F0FSYTKYZOQQjIvT19eHgwYNISUlBTk4Oh08Y4wAKEmOd01u3bkVkZCQ++eQTXLt2bUJnyYho2gP/r1ixYtp7P16vF8ePH0djYyOKioqwdOlS7mwOc/y/niCjUCiQk5ODxYsXo7q6GsnJycjLy4NSqbznxiyKonRxY1xcHDIzM6HRaHD58mVcvnw54CybUqmEwWCQzkyN7TUlJydPuWYigtVqRVNTE1JTU3nweCbhAApSsbGxePrpp3Hq1Cl88sknyM7ORlpa2rh7KT6fD+fPn8f8+fOxY8cOxMXFAbi1R1JbW4sTJ06AiJCYmIji4mKkpqaCiHDx4kVUVFTA4XBApVJNaW9lZGQEzc3N6O3tRXFxMQwGA+/1MAkHUJASBAFKpRJr167F4sWLUVdXh46ODuTl5SEhISFgI/f7/bDb7SguLg44za1Wq7Fx40ZcuXIFdrsdTz75JBYsWCC9NjMzE4Ig4NNPP8X8+fMnXNvYId8XX3yBc+fOITs7GwUFBbzXw+7CARTkBEGAXq9HcXEx+vr6UFlZiZiYGOTm5iIxMVEKqh//+MfjPiBRrVYjIyMDcXFxSE9PDwguQRCQkZGBBQsW4KGHHppQPW63Gx0dHbhw4QIefvhhPPvss9BoNLzXw8bFFyKGGL/fL/W3+Hw+rF+/HvPnz8fw8DD+/Oc/w+Vy3fWatWvXgojQ3t4+7ntevXoVv/rVr5CUlDTuciLC0NAQOjo60NPTg9TUVKxduxY6nY6DJ0xNdBvlAApBY9f59Pf3S3sjmZmZsFqtOHfuHIhICoaUlBQ899xz8Hg8+Nvf/nbX9UBJSUn4yU9+gqioKOk1Y38yLpcLFy5cwJkzZ6BQKJCRkYHly5dPub+IhQ6+EjqMjV3pbDQakZycjIKCAnR3d8NqtUKn08HtdsPn82HevHnYtm0btFotoqKiUFxcjP/9738YHh4GcKuju6SkBFFRUfD5fHC73RgYGMDly5dx4cIFqNVqpKWlYcuWLUhMTOQ719mk8R5QGCEieL1eDA4O4uzZs+jr60N/fz+io6MhCAKio6MxMjKCr7/+GjqdDgaDAXFxcRgZGYEoioiMjERUVBQyMjKQlJSE2NhYDh02Lt4DYncRBAFqtRrJyclITk4GEUEURYiiiKGhIbhcLni9XmRnZ0OtVoOIoNfrER0dLR1W3X4fGWPTxQEUxgRBQEREBCIiIvgudCYL3n9mjMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJpugvBVj7P5Zp9MpcyWMsfGMbZvfdq97UAbQ2ADraWlpMlfCGLufoaEhGAyGey4PygAau3Gyu7v7vl+OBXI6nUhLS0NPTw8PYzJBvM6mZmyUzJSUlPu2C8oAGhuDxmAw8B/FFOj1el5vk8TrbPImsnPAndCMMdlwADHGZBOUAaTRaPDuu+9Co9HIXUpQ4fU2ebzOHqygHBOaMRYagnIPiDEWGjiAGGOy4QBijMmGA4gxJpugDKD3338fCxcuRGRkJHJzc9HU1CR3SbIpLy9HTk4OYmJikJycjO3bt6OrqyugzejoKEpLS5GQkIDo6Gjs2LEDdrs9oE13dzdKSkqg0+mQnJyMt956Cz6fbza/imx+/etfQxAE7N69W5rH62yWUJDZt28fqdVq+utf/0qdnZ300ksvUWxsLNntdrlLk0VRURF9+OGHdPr0aWpvb6fvfOc7lJ6eTsPDw1KbnTt3UlpaGlVVVVFLSwvl5eXRhg0bpOU+n49WrFhBhYWFdPLkSTp06BAlJiZSWVmZHF9pVjU1NdHChQtp1apV9Nprr0nzeZ3NjqALoPXr11Npaan0s9/vp5SUFCovL5exqrmjr6+PAFBNTQ0REQ0ODpJKpaL9+/dLbc6ePUsAyGKxEBHRoUOHSKFQkM1mk9rs3buX9Ho9ud3u2f0Cs2hoaIgyMjKosrKSHn30USmAeJ3NnqA6BPN4PGhtbUVhYaE0T6FQoLCwEBaLRcbK5g6HwwHg/27YbW1thdfrDVhnmZmZSE9Pl9aZxWLBypUrYTQapTZFRUVwOp3o7OycxepnV2lpKUpKSgLWDcDrbDYF1c2o165dg9/vD/ilA4DRaMSXX34pU1VzhyiK2L17N/Lz87FixQoAgM1mg1qtRmxsbEBbo9EIm80mtRlvnY4tC0X79u1DW1sbmpub71rG62z2BFUAsfsrLS3F6dOnUVdXJ3cpc1pPTw9ee+01VFZWIjIyUu5ywlpQHYIlJiYiIiLirrMRdrsdJpNJpqrmhl27duHgwYM4evQoUlNTpfkmkwkejweDg4MB7W9fZyaTadx1OrYs1LS2tqKvrw/r1q2DUqmEUqlETU0N3nvvPSiVShiNRl5nsySoAkitViMrKwtVVVXSPFEUUVVVBbPZLGNl8iEi7Nq1CwcOHEB1dTUWLVoUsDwrKwsqlSpgnXV1daG7u1taZ2azGR0dHejr65PaVFZWQq/XY9myZbPzRWbR1q1b0dHRgfb2dmnKzs7G888/L/03r7NZIncv+GTt27ePNBoNffTRR3TmzBl6+eWXKTY2NuBsRDh55ZVXyGAw0LFjx8hqtUrTyMiI1Gbnzp2Unp5O1dXV1NLSQmazmcxms7R87JTytm3bqL29nSoqKigpKSmsTinffhaMiNfZbAm6ACIi+uMf/0jp6emkVqtp/fr11NDQIHdJsgEw7vThhx9KbVwuF7366qsUFxdHOp2OnnnmGbJarQHvc+nSJSouLiatVkuJiYn05ptvktfrneVvI587A4jX2ezg4TgYY7IJqj4gxlho4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbP4/lVSXRFYtBEcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Box(-1.0, 1.0, (1,), float32)\n",
      "state= [-0.5616639  0.       ]\n",
      "action= [-0.5101635]\n",
      "next_state= [-5.6214428e-01 -4.8037665e-04]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.sac.sac.SAC at 0x7f739b0fb400>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import SAC\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = SAC(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    buffer_size=10_0000,  #reply_buffer_size\n",
    "    learning_starts=100,  #积累N步的数据以后开始训练\n",
    "    batch_size=256,  #每次采样N条数据\n",
    "    tau=5e-3,  #target网络软更新系数\n",
    "    gamma=0.9,\n",
    "    train_freq=(1, 'step'),  #训练频率\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/7.SAC.Mountain Car Continuous')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-151.55, 118.81181548987459)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = SAC.load('save/7.SAC.Mountain Car Continuous')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhg0lEQVR4nO3da2xT9/0/8PexYztXxyE3ExJDyqWBcs/VhNICKSljpVB2FaqqbmpFG6bSVkiN9C/VpEmptidbt4o+mNb2wTQ2prGtDNKGBBJCnIsTUkKgadNCY0jsBBLbuTi+nc//Ac354UJpAomPL5+XdKT6nK/tj084757zPed8j0BEBMYYk4FC7gIYY9GLA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhvZAujdd9/FokWLEBsbi+LiYrS2tspVCmNMJrIE0N///ne89tpreOutt9DR0YE1a9agvLwcg4ODcpTDGJOJIMfNqMXFxSgsLMSf/vQnAIAoisjJycGvfvUrvPHGG8EuhzEmk5hgf6HH40F7ezsqKyuleQqFAmVlZTCZTHd9j9vthtvtll6Loojh4WGkpqZCEIQ5r5kxNjNEhNHRUWRlZUGh+O4DraAH0I0bN+D3+5GZmRkwPzMzE5999tld31NVVYVf//rXwSiPMTaLLBYLsrOzv3N50APoflRWVuK1116TXjscDhgMBlgsFmi1WhkrY4zdjdPpRE5ODpKSku7ZLugBlJaWBqVSCZvNFjDfZrNBr9ff9T0ajQYajeaO+VqtlgOIsRD2fV0kQT8LplarkZ+fj9raWmmeKIqora2F0WgMdjmMMRnJcgj22muv4bnnnkNBQQGKiorw+9//HuPj43j++eflKIcxJhNZAuinP/0phoaGcOjQIVitVqxduxbV1dV3dEwzxiKbLNcBPSin04nk5GQ4HA7uA2IsBE13Gw2Ls2CMsbnn8VyH3z8ClSoHSmU8gJg5v86OA4gxBgCw2/+N/v7/h5iYDMTG5iE+fg3i4lYjNvYRqFSZUCqTMXXearaCiQOIMfYNEX6/HX6/HW7353A4/gsgBgpFLNTqHGg0S5CQUAStthwJCYWz8o0cQIyx76BATIwOMTHzERf3iLRHpFJlzdo3cAAxxgAAXm8MYmPXIi5u+Tdhsw4aTS5UqiwoFLEAFLPeJ8QBxBgDAHR0pGPDhv9g3rwFEARlUL6TR0RkjAEAvF4FBCExaOEDcAAxFvWICD6fD2NjY+ju7kYwLw3kQzDGohARwe/3Y2BgAE1NTaiursYnn3wCjUaDxsZGZGXNXkfzvXAAMRYliAhutxsWiwUNDQ346KOP0N7eDqvVCp/PBwCIiYlBS0sLdu/eHZSaOIAYi2BEhMnJSfT29qKurg7V1dVob2/HjRs37nqo5fP5cPz4cezcuRNK5dz3BXEAMRZhiAhOpxOXLl1CTU0Nampq0NXVBafTOa3+nbq6OjidTqSkpMx5rRxAjEWY06dP4+DBg+jp6cH4+Pi03iMIAgRBABFhYGAAnZ2d2Lx58xxXygHEWMRJSUnBF198Me3wyc3NRVFREUZGRiAIAtrb23HixAk8/vjjc34zKp+GZyzCLF26FCtXrpxW24ULF+JHP/oR8vLyoNPpsHbtWuzZswdmsxlOp3OOK+U9IMYiTkJCAp544omAx1zp9XosXrwYgiCgt7cXNpsNSqUSW7ZsQUJCQsD79Xp9UPp/AN4DYiziCIKAzZs3S2exVqxYgWeffRZlZWXYunUrnn32WSxfvhwKheKug4UJgoC1a9ciOTl5zmvlPSDGItC6deuwePFiDA0Noby8PGAvJyEhAeXl5cjKykJmZiY8Hs8d78/IyAh4ffvZMyIK6Bt6kH4iDiDGIlBSUhLKy8vx8ccf33UvR6vV4p133oFOp8M//vEP2O12adnChQuRl5eH69evY2BgAENDQ5icnITD4UBcXBzGxsYQFxcHlUqF2NhYzJs3D4mJiViwYAGSkpKgVqunXScHEGMRSKFQYMeOHTh//jwUCsUd1/8olUpoNBro9Xr85Cc/QUtLC7766it4vV5MTk7if//7H/R6PdLT07FkyRKkpKRApVIFPGZZFEVMTk5ieHgYAwMDuHLlChwOBxISEqbdh8SD0jMWobxeL2w2G5qamnD58uWAZcuXL8fOnTtx48YNnD9/Hjdu3IBKpUJRUREMBkPAg0Cnc4h1e4zY7Xb09PTAaDR+7zbKAcRYhBsdHUVtbS2+/PJLAEB2djZycnLQ09MDvV6P1atXIyMjAzExszcIPT8VgzEG4FZ/0FNPPQWHw4Guri589dVXUCqV+PGPf4zExERZa+MAYizC+f1+dHd3o6urC8uXL8fevXuhUqnm/Crn6eAAYixCERGuX7+OtrY2pKWlYdeuXYiPjw+J4JnCAcRYBPJ4PGhtbYXNZsOWLVug0+lCKnimcAAxFkGICP39/Th9+jQKCgpgNBqDMq7P/eIAYixC+P1+dHZ2wmazYefOnUhKSgrJvZ7bzfhesIaGBjz11FPIysqCIAj497//HbCciHDo0CHMnz8fcXFxKCsrwxdffBHQZnh4GHv37oVWq4VOp8Mvf/lLjI2NPdAPYSyajY+P4+OPP4YgCCgvL4dWqw358AHuI4DGx8exZs0avPvuu3dd/tvf/hbvvPMO3nvvPbS0tEj3nUxOTkpt9u7di+7ubtTU1OD48eNoaGjAiy++eP+/grEoRUSwWCw4deoU1q1bh3Xr1oX0Idcd6AEAoGPHjkmvRVEkvV5Pv/vd76R5drudNBoN/e1vfyMiokuXLhEAamtrk9qcPHmSBEGg69evT+t7HQ4HASCHw/Eg5TMW1kRRpM8++4z++c9/0tjYGImiKHdJkuluo7M6HMeVK1dgtVpRVlYmzUtOTkZxcbE0NonJZIJOp0NBQYHUpqysDAqFAi0tLXf9XLfbDafTGTAxFs18Ph9MJhOGhoawc+dOJCQkhMUh17fNagBZrVYAQGZmZsD8zMxMaZnVar3jVv+YmBjMmzdPavNtVVVVSE5OlqacnJzZLJuxsOL1enH27FmoVCqUlpZCpVLJXdJ9C4sBySorK+FwOKTJYrHIXRJjQUffPGKnvr4eBoMBBQUFYbnXc7tZDSC9Xg8AsNlsAfNtNpu0TK/XY3BwMGC5z+fD8PCw1ObbNBoNtFptwMRYNKFvHipYU1ODhx9+GA899FDYhw8wywGUm5sLvV6P2tpaaZ7T6URLSwuMRiMAwGg0wm63o729XWpTV1cHURRRXFw8m+UwFjHGx8dRV1eHkpISZGdnR0T4APdxIeLY2Bh6e3ul11euXEFnZyfmzZsHg8GAAwcO4De/+Q2WLl2K3NxcvPnmm8jKysKuXbsA3BqH5Mknn8QLL7yA9957D16vF/v378fPfvazoD2PmrFwQUQYGxtDbW0tNm3ahHnz5sld0uya6em106dPE4A7pueee46Ibp0afPPNNykzM5M0Gg1t3bqVenp6Aj7j5s2b9POf/5wSExNJq9XS888/T6Ojo9OugU/Ds2jhdDrp5MmTNDIyElKn2b/PdLdRHpCMsRBERLDb7Th37hw2btwInU4nd0kzMt1tNCzOgjEWTYgIExMTOHfuHIxGY1AejyMXvhmVsRAzOTmJM2fOYMOGDZHX5/MtvAfEWAhxu92ora1FQUFB0J5OKicOIMZCxNQVzmvXrkVGRkbEnGq/Fw4gxkKA1+tFbW0tHnroISxYsCAqwgfgAGJMdkSE1tZW6PV65ObmRk34ABxAjMmKiNDV1QW1Wo01a9ZEVfgAHECMyYaIcPHiRfT39yM/Pz/qwgfgAGJMNgMDA+jv78eWLVsCnrkeTaLzVzMms6kbsjdt2gS1Wi13ObLhAGIsyFwuF06cOAGj0YjY2Fi5y5FVWF8JHYa3sbEo5/f70dDQgMceewxpaWlylyO7sN4DcjqdHEIsbIiiCJPJhOzsbB565hthHUBNTU3weDxyl8HY9yIiXL58GR6PBytWrIjKM153E9YBlJeXh9raWvh8PrlLYew7ERFsNhssFgseffRRDp/bhHUALVq0CGlpaejs7ORDMRayxsbGYDKZsGnTprB+gsVcCOsAEgQB+fn5cDqd+PrrrzmEWMjxer04c+YMCgsLERcXJ3c5ISesAwgAlEolNm7cCLPZjJGREbnLYUwiiiKam5uxbNmyqLrBdCbCPoAAQKVS4fHHH8fZs2fhdrvlLocx6TYLhUKBpUuXcvh8h4gIIEEQkJqailWrVqGpqQmiKMpdEotiRIShoSFcuXIFxcXFUXubxXREzJoRBAG5ublISEjA+fPnuT+IycblcqG5uRlbtmxBTExYX+s75yImgIBbIbR+/XpYrVZYrVYOIRZ0Pp8PdXV1WL9+PZKSkuQuJ+RFVAABQExMDDZv3oyWlha4XC65y2FRhIhgNpuRnp6OBQsWyF1OWIi4AAKAuLg4FBYW4tSpU/B6vXKXw6IAEeHatWsYHR1FQUEBdzpPU0QGkCAIyMrKQlZWFvcHsaBwOp1obW3Fo48+CqVSKXc5YSMiAwi4FULr1q3D6Ogo+vr65C6HRTCfz4fa2loUFxdDo9HIXU5YidgAAm5dpFhaWgqz2Yzh4WHeE2KzbqrfZ9myZcjOzuZDrxmK6AACAI1GA6PRiPr6evj9frnLYRHmyy+/xMTEBFasWCF3KWFpRgFUVVWFwsJCJCUlISMjA7t27UJPT09Am8nJSVRUVCA1NRWJiYnYs2cPbDZbQJu+vj7s2LED8fHxyMjIwMGDB+fsjnZBEDB//nwsXboUZrOZ94LYrLHb7WhtbUVJSQlfbHifZrTW6uvrUVFRgebmZtTU1MDr9WLbtm0YHx+X2rz66qv46KOPcPToUdTX16O/vx/PPPOMtNzv92PHjh3weDxoamrChx9+iA8++ACHDh2avV/1LYIg4JFHHsHY2Bi+/PJLDiH2wLxeLxobG/HEE08gPj5e7nLCFz2AwcFBAkD19fVERGS320mlUtHRo0elNpcvXyYAZDKZiIjoxIkTpFAoyGq1Sm0OHz5MWq2W3G73tL7X4XAQAHI4HDOqd2Jigv7617+S0+mc0fsYu50oinTu3Dnq7u4mURTlLickTXcbfaD9RofDAQCYN28eAKC9vR1erxdlZWVSm7y8PBgMBphMJgCAyWTCqlWrkJmZKbUpLy+H0+lEd3f3Xb/H7XbD6XQGTPcjLi4OZWVlaGho4EHM2H0hInzxxRe4efMm8vLyuNP5Ad13AImiiAMHDqC0tBQrV64EAFitVqjVauh0uoC2mZmZsFqtUpvbw2dq+dSyu6mqqkJycrI05eTk3G/ZSE9PR05ODvcHsfsyOjqKS5cu4YknnuB+n1lw32uwoqICFy9exJEjR2aznruqrKyEw+GQJovFct+fJQgCVq5ciaGhIVy9epVDiE2bz+dDQ0MDSktLo/5xOrPlvgJo//79OH78OE6fPo3s7Gxpvl6vh8fjgd1uD2hvs9mg1+ulNt8+Kzb1eqrNt2k0Gmi12oDpQSgUCpSVleHChQsYGxt7oM9i0YGI0NDQgKysLH6cziyaUQAREfbv349jx46hrq4Oubm5Acvz8/OhUqlQW1srzevp6UFfXx+MRiMAwGg0oqurC4ODg1KbmpoaaLXaoF5LERcXh+LiYjQ2NnJ/ELsnIkJfXx98Ph/Wrl3L/T6zaSY92y+99BIlJyfTmTNnaGBgQJomJiakNvv27SODwUB1dXVkNpvJaDSS0WiUlvt8Plq5ciVt27aNOjs7qbq6mtLT06mysnLWe9i/jyiK1N7eTufOneOzGew7jY6O0r/+9S9yuVxylxI2pruNziiAANx1ev/996U2LpeLXn75ZUpJSaH4+HjavXs3DQwMBHzO1atXafv27RQXF0dpaWn0+uuvk9frnfUfNx0+n48++eQT6uvre+DPYpHH4/HQ0aNHqb+/X+5Swsp0t1GBKPx6YZ1OJ5KTk+FwOB64Pwi4NYJddXU1tm3bhvj4eN7FZgBuHXq1t7dLA93xv4vpm+42yucRAcTGxqKoqAinTp3i8aSZ5Nq1axgZGeF+nznEAYT/Gz8oOzsbn376KZ+aZ3A6nWhpaUFpaSmP7zOHOIC+IQgC1q5dC5vNhoGBAbnLYTLy+/1obGxESUkJP0xwjnEA3UapVGLTpk1obGwMuMGWRQ8iQmdnJ+bPn88PEwwCDqBviY+Pl64P4v6g6EJEsFgsGBoawurVqzl8goAD6FsEQYDBYEBqaiq6urq4PyiKuFwutLS08LjOQcQBdBdT40lfu3YN169f5xCKAn6/Hw0NDdiwYQMSEhLkLidqcAB9B6VSiY0bN6K1tZWfLxbhiAiffvopMjIykJWVJXc5UYUD6B6Sk5NRWFiIxsZGHk86QhER+vv7ce3aNaxZs4b7fYKMA+h7ZGdnIzU1FR0dHXwoFoFcLhfa2tqwdetWHt9HBrzGv4cgCFizZg2sViv6+/s5hCKIz+fD2bNnUVRUhISEBN77kQEH0DTExMRg69at3B8UQeib53mlpqZi/vz5cpcTtTiApmlq/KAzZ87w+EERwGKxYGxsDOvWreM9HxlxAE3T1PPF0tPT+XnzYYyIMDo6io6ODmzcuJGv95EZB9AMTA3L4HQ6+XnzYcrv9+PEiRMoKSnh57iHAA6gGZp63nxnZydGRkZ4TyiMiKKI1tZWrFq1Cnq9ng+9QgAH0H3QaDQoLS1FfX099weFCSJCb28vfD4f8vLy5C6HfYMD6D4IgoDU1FQ88sgjMJlMfNNqiCMiDA4Owmw283PcQwz/Je6TIAhYsmQJlEoluru7+VAshLndbjQ3N+OHP/wh1Gq13OWw23AAPQBBEFBUVITPP/+cBzELUX6/H2fPnsX69euRlJQkdznsWziAHpBKpcKTTz6JtrY2fshhiJkaVF6n0yE7O5s7nUMQB9AsiI+Ph9FoRH19Pbxer9zlMNwKn6+++goul4ufaBHCOIBmgSAISE9Px6JFi9Dc3Myd0jIjIty8eROXLl3Chg0b+GLDEMYBNEsEQcCKFSug0Wi4U1pmLpcLH3/8MTZu3IiYmBi5y2H3wAE0iwRBQEFBASwWC65evcohJAOfz4eGhgZs2bIFKSkpfOgV4jiAZplCocBjjz2G8+fP85XSQSaKIpqamrBo0SLo9Xq5y2HTwAE0BxISElBWVobGxkZMTk7KXU5UICJ0dXVBqVTi4Ycf5j2fMDGjADp8+DBWr14NrVYLrVYLo9GIkydPSssnJydRUVGB1NRUJCYmYs+ePbDZbAGf0dfXhx07diA+Ph4ZGRk4ePBgRN7OkJSUhPz8fFRXV8Pj8chdTkQjIly9ehXDw8MoKSnh8AkjMwqg7OxsvP3222hvb4fZbMaWLVvw9NNPo7u7GwDw6quv4qOPPsLRo0dRX1+P/v5+PPPMM9L7/X4/duzYAY/Hg6amJnz44Yf44IMPcOjQodn9VSFg6nHPy5cvR1NTE58ZmyNEBKvViu7ubn6McjiiB5SSkkJ//vOfyW63k0qloqNHj0rLLl++TADIZDIREdGJEydIoVCQ1WqV2hw+fJi0Wi253e5pf6fD4SAA5HA4HrT8OSeKInV0dJDZbCZRFOUuJ+I4nU76z3/+Q2NjY3KXwm4z3W30vvuA/H4/jhw5gvHxcRiNRrS3t8Pr9aKsrExqk5eXB4PBAJPJBAAwmUxYtWoVMjMzpTbl5eVwOp3SXlSkmRpTenh4GBcuXOBO6Vk0MTGBs2fP4tFHH+VneYWpGQdQV1cXEhMTodFosG/fPhw7dgwrVqyA1WqFWq2GTqcLaJ+ZmQmr1QoAsFqtAeEztXxq2Xdxu91wOp0BUzhRKBTYvHkzhoeH+fT8LHG73aiursaaNWvu+DfHwseMA+jhhx9GZ2cnWlpa8NJLL+G5557DpUuX5qI2SVVVFZKTk6UpJydnTr9vLsTExGDjxo24fPkyrFYrh9AD8Pv9aGxsxPr167FgwQLudA5jMw4gtVqNJUuWID8/H1VVVVizZg3+8Ic/QK/Xw+PxwG63B7S32WzSNRl6vf6Os2JTr+913UZlZSUcDoc0WSyWmZYdElQqFR5//HGYzWbcvHlT7nLC0lT4ZGdnY+HChXKXwx7QA18HJIoi3G438vPzoVKpUFtbKy3r6elBX18fjEYjAMBoNKKrqwuDg4NSm5qaGmi1WqxYseI7v0Oj0Uin/qemcBUfH4/Nmzejra0NIyMjcpcTVkRRhNlsRlxcHJYtW8Z7PpFgJj3bb7zxBtXX19OVK1fowoUL9MYbb5AgCPTJJ58QEdG+ffvIYDBQXV0dmc1mMhqNZDQapff7fD5auXIlbdu2jTo7O6m6uprS09OpsrJyTnrYQ5UoimS32+nIkSM0PDzMZ8emwe/3k9lsps7OTvL7/XKXw77HdLfRGQXQL37xC1q4cCGp1WpKT0+nrVu3SuFDRORyuejll1+mlJQUio+Pp927d9PAwEDAZ1y9epW2b99OcXFxlJaWRq+//jp5vd6ZlBH2ATTl5s2bVF1dTU6nU+5SQprf76empiY6d+4ch0+YmO42KhCFX2+o0+lEcnIyHA5HWB+OAcDIyAjOnj2LjRs38s2TdyGKIrq6uuDxeJCfn8/jOYeJ6W6j/NeUmU6ng9FoRFNTE0ZHR+UuJ6SIoojm5mZ4vV4OnwjFf1GZTQ1mVlpaiqamJgwPD/MpetwKn46ODgDA+vXrOXwiFP9VQ4ROp0NxcTFOnToV9afo/X4/mpuboVKp+DE6EY7/siFCEASkpKRg+/btaGtrw8DAQFTuCXm9XtTX10Oj0WD16tUcPhGO/7ohJjExEZs3b0ZHRwd6e3ujKoTGx8dRV1eHRYsW8UDyUYIDKMQIgoDY2Fhs27YNAwMDuHDhQsQP5UFEcDgcOHnyJJYvX47c3FwOnyjBARSiVCoVSktLMTExgTNnzkTs436ICF9//TXq6+uxZcsWGAwGDp8owgEUwpRKJUpKSmAwGFBTUwOn0xlRh2SiKOLixYtob2/Htm3bkJKSIndJLMg4gEKcIAhYvHgxCgoKUF1dDYvFEvYhRESYnJxEXV0dXC4Xnn76acTGxvKeTxTiAAoDU9cK7dy5Ez09PWhsbAzbcbSJCIODgzh+/DiysrJQWFjIz+6KYhxAYWKqc3rr1q2IjY3Ff//7X9y4cSOs9oa8Xi/Onj2LlpYWlJeXY/ny5bzXE+X4fz1hRqFQoLCwEEuWLEFdXR0yMjJQUlKCmJiYkN2YiQgDAwNobW1FdnY2Dx7PJHwzapgiIvj9fly4cAFXrlxBQUEBcnJyQu7CvYmJCbS1taG/vx/bt29HcnJyyAYlmz3T3UY5gMIcEWF0dBSNjY0QRRElJSVITU2VdSMnIvh8Pnz66af4/PPPUVBQgMWLF/NeTxThAIoyU527dXV1SEpKQnFxMdLS0oIeRG63G11dXejt7cXixYuxatUqaDQa3uuJMhxAUcrv90v9LT6fD0VFRViwYMGc9hFN7YV1dXXBYrEgOzsb69atQ3x8PAdPlOIAimJ0a6RLDA0NSXsjy5cvx7Jly5CRkQGFQvFAwTD1T8blcqG3txeXLl2CQqHA0qVL8cgjj0ClUnHwRDkOIAbgVli43W709fWhs7MTLpcLaWlpyMrKwsKFCxEXFycdIn1XaEz16bjdbgwPD+Prr79Gb28v1Go1cnJykJeXh7S0tJDrAGfy4QBidyAieDweDA4Oore3FyMjI7BarUhMTIQgCEhKSkJsbCx8Ph9iY2Nht9uhUCgwPj4OURQRGxuLhIQELF26FOnp6dDpdBw67K6mu43ydUBRRBAEaDQa5OTkICcnB0QEURQhiiJGR0fhcrng9XrhdruhVqtBRNBqtUhMTJQOq+61p8TYTHEARTFBEKBUKqFUKjFv3jy5y2FRiPefGWOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMkmLG/FmLp/1ul0ylwJY+xuprbN77vXPSwD6ObNmwCAnJwcmSthjN3L6OgokpOTv3N5WAbQ1I2TfX199/xxLJDT6UROTg4sFgsPYzJNvM7uz9QomVlZWfdsF5YBNDUGTXJyMv+juA9arZbX2wzxOpu56ewccCc0Y0w2HECMMdmEZQBpNBq89dZb0Gg0cpcSVni9zRyvs7kVlmNCM8YiQ1juATHGIgMHEGNMNhxAjDHZcAAxxmQTlgH07rvvYtGiRYiNjUVxcTFaW1vlLkk2VVVVKCwsRFJSEjIyMrBr1y709PQEtJmcnERFRQVSU1ORmJiIPXv2wGazBbTp6+vDjh07EB8fj4yMDBw8eBA+ny+YP0U2b7/9NgRBwIEDB6R5vM6ChMLMkSNHSK1W01/+8hfq7u6mF154gXQ6HdlsNrlLk0V5eTm9//77dPHiRers7KQf/OAHZDAYaGxsTGqzb98+ysnJodraWjKbzVRSUkIbNmyQlvt8Plq5ciWVlZXR+fPn6cSJE5SWlkaVlZVy/KSgam1tpUWLFtHq1avplVdekebzOguOsAugoqIiqqiokF77/X7KysqiqqoqGasKHYODgwSA6uvriYjIbreTSqWio0ePSm0uX75MAMhkMhER0YkTJ0ihUJDVapXaHD58mLRaLbnd7uD+gCAaHR2lpUuXUk1NDT322GNSAPE6C56wOgTzeDxob29HWVmZNE+hUKCsrAwmk0nGykKHw+EA8H837La3t8Pr9Qass7y8PBgMBmmdmUwmrFq1CpmZmVKb8vJyOJ1OdHd3B7H64KqoqMCOHTsC1g3A6yyYwupm1Bs3bsDv9wf80QEgMzMTn332mUxVhQ5RFHHgwAGUlpZi5cqVAACr1Qq1Wg2dThfQNjMzE1arVWpzt3U6tSwSHTlyBB0dHWhra7tjGa+z4AmrAGL3VlFRgYsXL6KxsVHuUkKaxWLBK6+8gpqaGsTGxspdTlQLq0OwtLQ0KJXKO85G2Gw26PV6maoKDfv378fx48dx+vRpZGdnS/P1ej08Hg/sdntA+9vXmV6vv+s6nVoWadrb2zE4OIj169cjJiYGMTExqK+vxzvvvIOYmBhkZmbyOguSsAogtVqN/Px81NbWSvNEUURtbS2MRqOMlcmHiLB//34cO3YMdXV1yM3NDVien58PlUoVsM56enrQ19cnrTOj0Yiuri4MDg5KbWpqaqDVarFixYrg/JAg2rp1K7q6utDZ2SlNBQUF2Lt3r/TfvM6CRO5e8Jk6cuQIaTQa+uCDD+jSpUv04osvkk6nCzgbEU1eeuklSk5OpjNnztDAwIA0TUxMSG327dtHBoOB6urqyGw2k9FoJKPRKC2fOqW8bds26uzspOrqakpPT4+qU8q3nwUj4nUWLGEXQEREf/zjH8lgMJBaraaioiJqbm6WuyTZALjr9P7770ttXC4Xvfzyy5SSkkLx8fG0e/duGhgYCPicq1ev0vbt2ykuLo7S0tLo9ddfJ6/XG+RfI59vBxCvs+Dg4TgYY7IJqz4gxlhk4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbP4/bA0avzQZma8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-166.0 267 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
